package core_sql.day03

import java.net.URL

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by root on 2019/3/10.
  */
object FavTeacher {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf()
      conf.setMaster("local[*]").setAppName("FavTeacher")

    val sc: SparkContext = new SparkContext(conf)

    val lines: RDD[String] = sc.textFile("src/main/file/teacher.log")

    val teacherAndOne: RDD[(String, Int)] = lines.map(line => {
      val index = line.lastIndexOf("/") + 1
      val teacher: String = line.substring(index)

      val host: String = new URL(line).getHost
      val subject = host.substring(0, host.indexOf("."))
      (subject + "_" + teacher, 1)
    })
    // 聚合
    val reduced: RDD[(String, Int)] = teacherAndOne.reduceByKey(_+_)
    // 排序
    val result: RDD[(String, Int)] = reduced.sortBy(_._2,false)

    val collect: Array[(String, Int)] = result.collect()

    println(collect.toBuffer)

    sc.stop()

  }

}
